#define USE_UNICODE

#include "LexMap.h"
#include "Tokenizer.h"
#include "Compilation.h"

Token * Tokenizer::tokenize(const char *p, Token **end_token) {
    Token head = {};
    Token *cur = &head;

    // Skip the UTF-8 "BOM" marker
    if (!memcmp(p, "\xef\xbb\xbf", 3)) p += 3;

    while (*p) {
        // Skip whitespace characters
        // 
        while (IS_LEX_ANY_SPACE(*p)) {
            p++;
        }

        // Numeric literals
        // E.g. 123  1e3  1.2  .12  ,12  123'45  123'45E+2  1,23e-2
        if (IS_LEX_NUMBER(*p) || ((*p == '.' || *p == ',') && IS_LEX_NUMBER(p[1]))) {
            TokenId id = IS_LEX_NUMBER(*p) ? TokenIdInteger : TokenIdFloat;
            bool has_dot = (id == TokenIdFloat);
            bool has_exp = false;
            const char *start = p++;
            while (Lex_Map[(*p)] > (LEX_SPECIAL|LEX_SPECIAL_COLON)) {
                if (IS_LEX_NUMBER(*p) || *p == '\'') {
                    p++;
                } else if (*p == 'e' || *p == 'E') {
                    if (has_exp) tokenError(p, err_invalid_token);
                    has_exp = true;
                    id = TokenIdFloat;
                    p++;
                    if (*p == '-' || *p == '+') p++;
                } else if (*p == '.' || *p == ',') {
                    if (has_exp || has_dot)
                        tokenError(p, err_invalid_token);
                    has_dot = true;
                    id = TokenIdFloat;
                    p++;
                } else tokenError(p, err_invalid_token);
            }
            cur = cur->next = newToken(comp.tokBuf, id, start, p - start);
            continue;
        }

        if (*p) tokenError(p, err_invalid_token);
    }

    *end_token = cur;
    return head.next;
}

template <typename... Args>
Token * Tokenizer::tokenError(const char *loc, unsigned diag_id, Args &&... args) {
    comp.error(loc, diag_id, std::forward<Args>(args)...);
    return newToken(comp.tokBuf, TokenIdUnknown, loc, 0);
}

void dumpTokens(Token *tok, const char *msg) {if constexpr (verbose >= 3) {
    llvm::outs() << "\n" << msg << "\n";
    int n = 10;
    while (tok) {
        llvm::outs() << TokenNames[tok->id] << ":" << tok->str() << " ";
        if (!n--) {
            n = 10;
            llvm::outs() << '\n';
        }
        tok = tok->next;
    }
    llvm::outs() << "\n\n";
}}